package net.bwie.lianxi.job;

import com.alibaba.fastjson.JSON;
import net.bwie.lianxi.function.IsNewProcessFunction;
import net.bwie.lianxi.function.splitProcessFunction;
import net.bwie.realtime.jtp.utils.KafkaUtil;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.datastream.SideOutputDataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;


public class JtpAppLogEtlJob {

    public static void main(String[] args) throws  Exception{

        // 执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //读取数据源
        DataStream<String> stream = KafkaUtil.consumerKafka(env, "topic-log");
//        stream.print();

        // 数据转换
        DataStream<String> pageStream = processAppLog(stream);

        //数据输出
        KafkaUtil.producerKafka(pageStream,"dwd-traffic-page-log");

        env.execute("JtpAppLogEtlJob");
    }

    /**
     * 1.数据清洗
     * 2.新老访客状态标记修复
     * 3.数据分流
     * @param stream
     * @return
     */
    private static DataStream<String>  processAppLog(DataStream<String> stream) {

        //1.数据清洗
        DataStream<String> jsonStream = CleanedAppLog(stream);

        //2.新老访客状态标记修复
        DataStream<String> IsNewStream = IsNewAppLog(jsonStream);

        //3.数据分流
        DataStream<String> splitStream = splitAppLog(IsNewStream);

        return splitStream;
    }

    private static DataStream<String> splitAppLog(DataStream<String> stream) {

        // 1.侧边流输出标记
        OutputTag<String> errorTag = new OutputTag<String>("error-log") {
        };
        OutputTag<String> startTag = new OutputTag<String>("start-log") {
        };
        OutputTag<String> displaysTag = new OutputTag<String>("displays-log") {
        };
        OutputTag<String> actionTag = new OutputTag<String>("action-log") {
        };

        // 2.日志分流处理
        SingleOutputStreamOperator<String> pageStream = stream.process(
                new splitProcessFunction(errorTag, startTag, displaysTag, actionTag)
        );

        SideOutputDataStream<String> errorStream = pageStream.getSideOutput(errorTag);
        KafkaUtil.producerKafka(errorStream,"dwd-traffic-error-log");
        SideOutputDataStream<String> startStream = pageStream.getSideOutput(startTag);
        KafkaUtil.producerKafka(startStream,"dwd-traffic-start-log");
        SideOutputDataStream<String> displayStream = pageStream.getSideOutput(displaysTag);
        KafkaUtil.producerKafka(displayStream,"dwd-traffic-display-log");
        SideOutputDataStream<String> actionStream = pageStream.getSideOutput(actionTag);
        KafkaUtil.producerKafka(actionStream,"dwd-traffic-action-log");

        return pageStream;
    }

    private static DataStream<String> IsNewAppLog(DataStream<String> stream) {
        // 1.按照设备id进行分组
        KeyedStream<String, String> midStream = stream.keyBy(new KeySelector<String, String>() {
            @Override
            public String getKey(String value) throws Exception {
                return JSON.parseObject(value).getJSONObject("common").getString("mid");
            }
        });

        // 2.状态编程
        SingleOutputStreamOperator<String> isNewStream = midStream.process(new IsNewProcessFunction());

        // 3.返回数据流
        return isNewStream;
    }

    private static DataStream<String> CleanedAppLog(DataStream<String> stream) {
        // 侧边流输出标记
        final OutputTag<String> dirtyTag = new OutputTag<String>("dirty-log"){};

        // 数据清洗
        SingleOutputStreamOperator<String> cleanedStream = stream.process(new ProcessFunction<String, String>() {
            @Override
            public void processElement(String value, Context ctx, Collector<String> out) throws Exception {
                try {
                    //解析json数据
                    JSON.parseObject(value);
                    out.collect(value);
                }catch (Exception e){
                    ctx.output(dirtyTag,value);
                }
            }
        });

        // 侧边流输出：脏数据
        SideOutputDataStream<String> sideOutput = cleanedStream.getSideOutput(dirtyTag);
        KafkaUtil.producerKafka(sideOutput,"dwd-traffic-dirty-log");

        // 返回正常数据
        return cleanedStream;
    }

}
